<!DOCTYPE html>





<html class="theme-next muse use-motion" lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="generator" content="Hexo 3.9.0">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png?v=7.3.0">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png?v=7.3.0">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png?v=7.3.0">
  <link rel="mask-icon" href="/images/logo.svg?v=7.3.0" color="#222">

<link rel="stylesheet" href="/css/main.css?v=7.3.0">


<link rel="stylesheet" href="/lib/font-awesome/css/font-awesome.min.css?v=4.7.0">


<script id="hexo-configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Muse',
    version: '7.3.0',
    exturl: false,
    sidebar: {"position":"left","display":"post","offset":12,"onmobile":false},
    back2top: {"enable":true,"sidebar":false,"scrollpercent":false},
    save_scroll: false,
    copycode: {"enable":false,"show_result":false,"style":null},
    fancybox: false,
    mediumzoom: false,
    lazyload: false,
    pangu: false,
    algolia: {
      appID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    },
    localsearch: {"enable":false,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},
    path: '',
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    translation: {
      copy_button: '复制',
      copy_success: '复制成功',
      copy_failure: '复制失败'
    }
  };
</script>

  <meta name="description" content="爬虫种类Spider start_urls = []   URL列表  start_requests()   该方法必须返回一个可迭代对象(iterable),包含了spider用于爬取的第一个Request,  该方法仅仅会被Scrapy调用一次.   作用：需要在启动时以POST登录某个网站，可以重写该方法   例：   123456789def start_requests(self):">
<meta name="keywords" content="爬虫">
<meta property="og:type" content="article">
<meta property="og:title" content="Scrapy 教程">
<meta property="og:url" content="http://yoursite.com/2018/04/21/Scrapy-教程/index.html">
<meta property="og:site_name" content="XuQi&#39;s Blog">
<meta property="og:description" content="爬虫种类Spider start_urls = []   URL列表  start_requests()   该方法必须返回一个可迭代对象(iterable),包含了spider用于爬取的第一个Request,  该方法仅仅会被Scrapy调用一次.   作用：需要在启动时以POST登录某个网站，可以重写该方法   例：   123456789def start_requests(self):">
<meta property="og:locale" content="zh-CN">
<meta property="og:updated_time" content="2019-10-20T13:23:23.041Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Scrapy 教程">
<meta name="twitter:description" content="爬虫种类Spider start_urls = []   URL列表  start_requests()   该方法必须返回一个可迭代对象(iterable),包含了spider用于爬取的第一个Request,  该方法仅仅会被Scrapy调用一次.   作用：需要在启动时以POST登录某个网站，可以重写该方法   例：   123456789def start_requests(self):">
  <link rel="canonical" href="http://yoursite.com/2018/04/21/Scrapy-教程/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome: false,
    isPost: true,
    isPage: false,
    isArchive: false
  };
</script>

  <title>Scrapy 教程 | XuQi's Blog</title>
  








  <noscript>
  <style>
  .use-motion .motion-element,
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-title { opacity: initial; }

  .use-motion .logo,
  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-CN">

  <div class="container sidebar-position-left">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta">

    <div class="custom-logo-site-title">
      <a href="/" class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">XuQi's Blog</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
  </div>

  <div class="site-nav-toggle">
    <button aria-label="切换导航栏">
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>


<nav class="site-nav">
  
  <ul id="menu" class="menu">
      
      
      
        
        <li class="menu-item menu-item-home">
      
    

    <a href="/" rel="section"><i class="menu-item-icon fa fa-fw fa-home"></i> <br>首页</a>

  </li>
      
      
      
        
        <li class="menu-item menu-item-archives">
      
    

    <a href="/archives/" rel="section"><i class="menu-item-icon fa fa-fw fa-archive"></i> <br>归档</a>

  </li>
  </ul>

    

</nav>
</div>
    </header>

    


    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content page-post-detail">
            

  <div id="posts" class="posts-expand">
    

  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2018/04/21/Scrapy-教程/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="XuQi">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="XuQi's Blog">
    </span>
      <header class="post-header">

        
          <h1 class="post-title" itemprop="name headline">Scrapy 教程

            
          </h1>
        

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
                
              

              <time title="创建时间：2018-04-21 12:47:16" itemprop="dateCreated datePublished" datetime="2018-04-21T12:47:16+00:00">2018-04-21</time>
            </span>
          
            

            
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2019-10-20 13:23:23" itemprop="dateModified" datetime="2019-10-20T13:23:23+00:00">2019-10-20</time>
              </span>
            
          

          

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <h3 id="爬虫种类"><a href="#爬虫种类" class="headerlink" title="爬虫种类"></a>爬虫种类</h3><h5 id="Spider"><a href="#Spider" class="headerlink" title="Spider"></a>Spider</h5><ul>
<li><p>start_urls = []</p>
<p>  URL列表</p>
</li>
<li><p>start_requests()</p>
<p>  该方法必须返回一个可迭代对象(iterable),包含了spider用于爬取的第一个Request,<br>  该方法仅仅会被Scrapy调用一次.</p>
<p>  <strong>作用：</strong>需要在启动时以POST登录某个网站，可以重写该方法</p>
<p>  例：</p>
  <figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">def start_requests(self):</span><br><span class="line">   return [scrapy.FormRequest(&quot;http://www.example.com/login&quot;,</span><br><span class="line">                              formdata=&#123;&apos;user&apos;: &apos;john&apos;, &apos;pass&apos;: &apos;secret&apos;&#125;,</span><br><span class="line">                              callback=self.logged_in)]</span><br><span class="line"></span><br><span class="line">def logged_in(self, response):</span><br><span class="line">    # here you would extract links to follow and return Requests for</span><br><span class="line">    # each of them, with another callback</span><br><span class="line">    pass</span><br></pre></td></tr></table></figure>


</li>
</ul>
<ul>
<li><p>make_requests_from_url(url)</p>
<p>  默认未被复写(overridden)的情况下,该方法返回的Request对象中， parse() 作为回调函数.</p>
</li>
<li><p>parse(response)</p>
<p>  该方法及其他的Request回调函数必须返回一个包含 Request 及(或) Item 的可迭代的对象.</p>
  <a id="more"></a>

</li>
</ul>
<h5 id="CrawlSpider"><a href="#CrawlSpider" class="headerlink" title="CrawlSpider"></a>CrawlSpider</h5><h5 id="XMLFeedSpider"><a href="#XMLFeedSpider" class="headerlink" title="XMLFeedSpider"></a>XMLFeedSpider</h5><h5 id="CSVFeedSpider"><a href="#CSVFeedSpider" class="headerlink" title="CSVFeedSpider"></a>CSVFeedSpider</h5><h5 id="SitemapSpider"><a href="#SitemapSpider" class="headerlink" title="SitemapSpider"></a>SitemapSpider</h5><h3 id="选择器-Selectors"><a href="#选择器-Selectors" class="headerlink" title="选择器(Selectors)"></a>选择器(Selectors)</h3><p>Scrapy提供了两个实用的快捷方式: response.xpath() 及 response.css(),方法返回一个类 SelectorList 的实例,为了提取真实的原文数据，你需要调用 .extract()<br>.re() 方法返回unicode字符串的列</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line">&lt;?xml version=&quot;1.0&quot; encoding=&quot;ISO-8859-1&quot;?&gt;</span><br><span class="line"></span><br><span class="line">&lt;bookstore&gt;</span><br><span class="line"></span><br><span class="line">&lt;book&gt;</span><br><span class="line">  &lt;title lang=&quot;eng&quot;&gt;Harry Potter&lt;/title&gt;</span><br><span class="line">  &lt;price&gt;29.99&lt;/price&gt;</span><br><span class="line">&lt;/book&gt;</span><br><span class="line"></span><br><span class="line">&lt;book&gt;</span><br><span class="line">  &lt;title lang=&quot;eng&quot;&gt;Learning XML&lt;/title&gt;</span><br><span class="line">  &lt;price&gt;39.95&lt;/price&gt;</span><br><span class="line">&lt;/book&gt;</span><br><span class="line"></span><br><span class="line">&lt;/bookstore&gt;</span><br></pre></td></tr></table></figure>

<ul>
<li>response.xpath()</li>
</ul>
<table>
<thead>
<tr>
<th>表达式</th>
<th>描述</th>
</tr>
</thead>
<tbody><tr>
<td>nodename</td>
<td>选取此节点的所有子节点。</td>
</tr>
<tr>
<td>/</td>
<td>从根节点选取。</td>
</tr>
<tr>
<td>//</td>
<td>从匹配选择的当前节点选择文档中的节点，而不考虑它们的位置。</td>
</tr>
<tr>
<td>.</td>
<td>选取当前节点。</td>
</tr>
<tr>
<td>..</td>
<td>选取当前节点的父节点。</td>
</tr>
<tr>
<td>@</td>
<td>选取属性。</td>
</tr>
<tr>
<td>*</td>
<td>匹配任何元素节点。</td>
</tr>
<tr>
<td>@*</td>
<td>匹配任何属性节点。</td>
</tr>
<tr>
<td>node()</td>
<td>匹配任何类型的节点。</td>
</tr>
</tbody></table>
<p>实例</p>
<table>
<thead>
<tr>
<th>路径表达式</th>
<th>结果</th>
</tr>
</thead>
<tbody><tr>
<td>bookstore</td>
<td>选取 bookstore 元素的所有子节点。</td>
</tr>
<tr>
<td>/bookstore</td>
<td>选取根元素 bookstore。注释：假如路径起始于正斜杠( / )，则此路径始终代表到某元素的绝对路径！</td>
</tr>
<tr>
<td>bookstore/book</td>
<td>选取属于 bookstore 的子元素的所有 book 元素。</td>
</tr>
<tr>
<td>//book</td>
<td>选取所有 book 子元素，而不管它们在文档中的位置。</td>
</tr>
<tr>
<td>bookstore//book</td>
<td>选择属于 bookstore 元素的后代的所有 book 元素，而不管它们位于 bookstore</td>
</tr>
<tr>
<td>//@lang</td>
<td>选取名为 lang 的所有属性。</td>
</tr>
<tr>
<td>/bookstore/book[1]</td>
<td>选取属于 bookstore 子元素的第一个 book 元素。</td>
</tr>
<tr>
<td>/bookstore/book[last()]</td>
<td>选取属于 bookstore 子元素的最后一个 book 元素。</td>
</tr>
<tr>
<td>/bookstore/book[last()-1]</td>
<td>选取属于 bookstore 子元素的倒数第二个 book 元素。</td>
</tr>
<tr>
<td>/bookstore/book[position()&lt;3]</td>
<td>选取最前面的两个属于 bookstore 元素的子元素的 book 元素。</td>
</tr>
<tr>
<td>//title[@lang]</td>
<td>选取所有拥有名为 lang 的属性的 title 元素。</td>
</tr>
<tr>
<td>//title[@lang=’eng’]</td>
<td>选取所有 title 元素，且这些元素拥有值为 eng 的 lang 属性。</td>
</tr>
<tr>
<td>/bookstore/book[price&gt;35.00]</td>
<td>选取 bookstore 元素的所有 book 元素，且其中的 price 元素的值须大于 35.00。</td>
</tr>
<tr>
<td>/bookstore/book[price&gt;35.00]/title</td>
<td>选取 bookstore 元素中的 book 元素的所有 title 元素，且其中的 price 元素的值须大于 35.00。</td>
</tr>
<tr>
<td>/bookstore/*</td>
<td>选取 bookstore 元素的所有子元素。</td>
</tr>
<tr>
<td>//*</td>
<td>选取文档中的所有元素。</td>
</tr>
<tr>
<td>//title[@*]</td>
<td>选取所有带有属性的 title 元素。</td>
</tr>
</tbody></table>
<p>//book/title | //book/price    选取 book 元素的所有 title 和 price 元素。</p>
<ul>
<li><p>response.css()</p>
</li>
<li><p>response.re()</p>
</li>
</ul>

    </div>

    
    
    
        
      

      <footer class="post-footer">
          
            
          
          <div class="post-tags">
            
              <a href="/tags/爬虫/" rel="tag"># 爬虫</a>
            
          </div>
        

        

          <div class="post-nav">
            <div class="post-nav-next post-nav-item">
              
                <a href="/2018/02/22/STM8的GPIO介绍/" rel="next" title="STM8_GPIO">
                  <i class="fa fa-chevron-left"></i> STM8_GPIO
                </a>
              
            </div>

            <span class="post-nav-divider"></span>

            <div class="post-nav-prev post-nav-item">
              
                <a href="/2018/04/25/Linux进程介绍/" rel="prev" title="Linux进程">
                  Linux进程 <i class="fa fa-chevron-right"></i>
                </a>
              
            </div>
          </div>
        
      </footer>
    
  </div>
  
  
  
  </article>

  </div>


          </div>
          

        </div>
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            站点概览
          </li>
        </ul>
      

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-overview">

          <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">XuQi</p>
  <div class="site-description motion-element" itemprop="description"></div>
</div>
  <nav class="site-state motion-element">
      <div class="site-state-item site-state-posts">
        
          <a href="/archives/">
        
          <span class="site-state-item-count">44</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
    
      
      
      <div class="site-state-item site-state-tags">
        
        
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
          
        
        <span class="site-state-item-count">30</span>
        <span class="site-state-item-name">标签</span>
        
      </div>
    
  </nav>



        </div>
      </div>
      <!--noindex-->
        <div class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
            
            
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-3"><a class="nav-link" href="#爬虫种类"><span class="nav-number">1.</span> <span class="nav-text">爬虫种类</span></a><ol class="nav-child"><li class="nav-item nav-level-5"><a class="nav-link" href="#Spider"><span class="nav-number">1.0.1.</span> <span class="nav-text">Spider</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#CrawlSpider"><span class="nav-number">1.0.2.</span> <span class="nav-text">CrawlSpider</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#XMLFeedSpider"><span class="nav-number">1.0.3.</span> <span class="nav-text">XMLFeedSpider</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#CSVFeedSpider"><span class="nav-number">1.0.4.</span> <span class="nav-text">CSVFeedSpider</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#SitemapSpider"><span class="nav-number">1.0.5.</span> <span class="nav-text">SitemapSpider</span></a></li></ol></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#选择器-Selectors"><span class="nav-number">2.</span> <span class="nav-text">选择器(Selectors)</span></a></li></ol></div>
            

          </div>
        </div>
      <!--/noindex-->
      

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; <span itemprop="copyrightYear">2019</span>
  <span class="with-love" id="animate">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">XuQi</span>
</div>
  <div class="powered-by">由 <a href="https://hexo.io" class="theme-link" rel="noopener" target="_blank">Hexo</a> 强力驱动 v3.9.0</div>
  <span class="post-meta-divider">|</span>
  <div class="theme-info">主题 – <a href="https://theme-next.org" class="theme-link" rel="noopener" target="_blank">NexT.Muse</a> v7.3.0</div>

        








        
      </div>
    </footer>
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
      </div>

    

  </div>

  
  <script src="/lib/jquery/index.js?v=3.4.1"></script>
  <script src="/lib/velocity/velocity.min.js?v=1.2.1"></script>
  <script src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>

<script src="/js/utils.js?v=7.3.0"></script>
  <script src="/js/motion.js?v=7.3.0"></script>


  <script src="/js/schemes/muse.js?v=7.3.0"></script>


<script src="/js/next-boot.js?v=7.3.0"></script>




  




























  

  

  


  
  <script src="/js/scrollspy.js?v=7.3.0"></script>
<script src="/js/post-details.js?v=7.3.0"></script>


</body>
</html>
